{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction\n",
    "In this notebook we will demonstrates how to use BigDL-Nano to accelerate PyTorch or PyTorch-Lightning applications on training workloads."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Environment\n",
    "Before you start with APIs delivered by bigdl-nano, you have to make sure BigDL-Nano is correctly installed for PyTorch. If not, please follow [this](../../../../../docs/readthedocs/source/doc/Nano/Overview/nano.md) to set up your environment.<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Cifar10 DataModule\n",
    "Import the existing data module from bolts and modify the train and test transforms.\n",
    "You could access [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) for a view of the whole dataset.\n",
    "Leveraging OpenCV and libjpeg-turbo, BigDL-Nano can accelerate computer vision data pipelines by providing a drop-in replacement of torch_vision's `datasets` and `transforms`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from torchvision.datasets import CIFAR10\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from bigdl.nano.pytorch.vision import transforms\n",
    "DATA_PATH = os.environ.get('DATA_PATH', '.')\n",
    "BATCH_SIZE = 64\n",
    "DEV_RUN = bool(os.environ.get('DEV_RUN', False))\n",
    "train_transforms = transforms.Compose(\n",
    "    [\n",
    "        transforms.RandomCrop(32, 4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.4913725490196078, 0.4823529411764706, 0.4466666666666667],\n",
    "                                     std=[0.24705882352941178, 0.24352941176470588, 0.2615686274509804])\n",
    "    ]\n",
    ")\n",
    "test_transforms = transforms.Compose(\n",
    "    [\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.4913725490196078, 0.4823529411764706, 0.4466666666666667],\n",
    "                                     std=[0.24705882352941178, 0.24352941176470588, 0.2615686274509804])\n",
    "    ]\n",
    ")\n",
    "train_dataset = CIFAR10(\n",
    "        root=DATA_PATH,\n",
    "        train=True,\n",
    "        transform=train_transforms,\n",
    "        download=True,\n",
    ")\n",
    "train_loader = DataLoader(\n",
    "        dataset=train_dataset,\n",
    "        batch_size=128\n",
    ")\n",
    "test_dataset = CIFAR10(\n",
    "    root=DATA_PATH,\n",
    "    train=False,\n",
    "    transform=test_transforms,\n",
    "    download=True\n",
    ")\n",
    "test_loader = DataLoader(\n",
    "        dataset=train_dataset,\n",
    "        batch_size=128\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Custom Model\n",
    "Modify the pre-existing Resnet architecture from TorchVision. The pre-existing architecture is based on ImageNet images (224x224) as input. So we need to modify it for CIFAR10 images (32x32)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim.lr_scheduler import OneCycleLR\n",
    "from torchvision.models import resnet18\n",
    "from pytorch_lightning import LightningModule, seed_everything\n",
    "from torchmetrics.functional import accuracy\n",
    "seed_everything(7)\n",
    "def create_model():\n",
    "    model = resnet18(pretrained=False, num_classes=10)\n",
    "    model.conv1 = nn.Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
    "    model.maxpool = nn.Identity()\n",
    "    return model\n",
    "\n",
    "class LitResnet(LightningModule):\n",
    "    def __init__(self, learning_rate=0.05, num_processes=1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.save_hyperparameters()\n",
    "        self.model = create_model()\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.model(x)\n",
    "        return F.log_softmax(out, dim=1)\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        logits = self(x)\n",
    "        loss = F.nll_loss(logits, y)\n",
    "        self.log(\"train_loss\", loss)\n",
    "        return loss\n",
    "\n",
    "    def evaluate(self, batch, stage=None):\n",
    "        x, y = batch\n",
    "        logits = self(x)\n",
    "        loss = F.nll_loss(logits, y)\n",
    "        preds = torch.argmax(logits, dim=1)\n",
    "        acc = accuracy(preds, y, 'multiclass', num_classes=10)\n",
    "\n",
    "        if stage:\n",
    "            self.log(f\"{stage}_loss\", loss, prog_bar=True)\n",
    "            self.log(f\"{stage}_acc\", acc, prog_bar=True)\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        self.evaluate(batch, \"val\")\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        self.evaluate(batch, \"test\")\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.SGD(\n",
    "            self.parameters(),\n",
    "            lr=self.hparams.learning_rate,\n",
    "            momentum=0.9,\n",
    "            weight_decay=5e-4,\n",
    "        )\n",
    "        steps_per_epoch = 45000 // BATCH_SIZE // self.hparams.num_processes\n",
    "        scheduler_dict = {\n",
    "            \"scheduler\": OneCycleLR(\n",
    "                optimizer,\n",
    "                0.1,\n",
    "                epochs=self.trainer.max_epochs,\n",
    "                steps_per_epoch=steps_per_epoch,\n",
    "            ),\n",
    "            \"interval\": \"step\",\n",
    "        }\n",
    "        return {\"optimizer\": optimizer, \"lr_scheduler\": scheduler_dict}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with Nano Apis\n",
    "The PyTorch Trainer (`bigdl.nano.pytorch.Trainer`) is the place where we integrate most optimizations. It extends PyTorch Lightning's Trainer and has a few more parameters and methods specific to BigDL-Nano. The Trainer can be directly used to train a `LightningModule`.\n",
    "\n",
    "`torch.channels_last` is recommended to be applied to the model object to raise CPU resource usage efficiency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.696    Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 391/391 [01:44<00:00,  3.75it/s, loss=0.221, v_num=6] \n",
      "49min 37s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "Testing: 100%|██████████| 391/391 [00:38<00:00, 11.70it/s]--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.905239999294281, 'test_loss': 0.27998998761177063}\n",
      "--------------------------------------------------------------------------------\n",
      "Testing: 100%|██████████| 391/391 [00:38<00:00, 10.19it/s]\n"
     ]
    }
   ],
   "source": [
    "from bigdl.nano.pytorch import Trainer\n",
    "model = LitResnet()\n",
    "trainer = Trainer(max_epochs=30,\n",
    "                  channels_last=True,\n",
    "                  fast_dev_run=DEV_RUN) # run model once quickly in test\n",
    "fit_time_basic = %timeit -n 1 -r 1 -o \\\n",
    "trainer.fit(model, train_dataloaders=train_loader)\n",
    "metric_basic = trainer.test(model, dataloaders=test_loader)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Intel Extension for Pytorch (a.k.a. IPEX) link extends PyTorch with optimizations for an extra performance boost on Intel hardware. BigDL-Nano integrates IPEX through the Trainer. Users can turn on IPEX by setting use_ipex=True."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torchvision/models/_utils.py:209: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n",
      "  f\"The parameter '{pretrained_param}' is deprecated since 0.13 and will be removed in 0.15, \"\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=None`.\n",
      "  warnings.warn(msg)\n",
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:532: LightningDeprecationWarning: `trainer.fit(train_dataloader)` is deprecated in v1.4 and will be removed in v1.6. Use `trainer.fit(train_dataloaders)` instead. HINT: added 's'\n",
      "  \"`trainer.fit(train_dataloader)` is deprecated in v1.4 and will be removed in v1.6.\"\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/configuration_validator.py:101: UserWarning: you defined a validation_step but have no val_dataloader. Skipping val loop\n",
      "  rank_zero_warn(f\"you defined a {step_name} but have no {loader_name}. Skipping {stage} loop\")\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.712    Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 1/391 [00:00<01:11,  5.42it/s, loss=2.35, v_num=5]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 391/391 [01:35<00:00,  4.10it/s, loss=0.231, v_num=5] \n",
      "46min 29s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-21 17:51:53,208 - root - WARNING - Convert model to channels last failed, fall back to origin memory format.Exception msg: required rank 4 tensor to use channels_last format\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing: 100%|█████████▉| 390/391 [00:34<00:00, 11.34it/s]--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.8967400193214417, 'test_loss': 0.3076363503932953}\n",
      "--------------------------------------------------------------------------------\n",
      "Testing: 100%|██████████| 391/391 [00:34<00:00, 11.32it/s]\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet()\n",
    "trainer = Trainer(max_epochs=30, \n",
    "                  use_ipex=True,\n",
    "                  fast_dev_run=DEV_RUN,\n",
    "                  channels_last=True)\n",
    "fit_time_ipex = %timeit -n 1 -r 1 -o \\\n",
    "trainer.fit(model, train_dataloaders=train_loader)\n",
    "metric_ipex = trainer.test(model, dataloaders=test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setting use_ipex=True will Apply optimizations at Python frontend to the given model (nn.Module), as well as the given optimizer (optional). Optimizations include conv+bn folding (for inference only), weight prepacking and so on."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Increase the number of processes on distributed training to accelerate training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torchvision/models/_utils.py:209: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n",
      "  f\"The parameter '{pretrained_param}' is deprecated since 0.13 and will be removed in 0.15, \"\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=None`.\n",
      "  warnings.warn(msg)\n",
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:532: LightningDeprecationWarning: `trainer.fit(train_dataloader)` is deprecated in v1.4 and will be removed in v1.6. Use `trainer.fit(train_dataloaders)` instead. HINT: added 's'\n",
      "  \"`trainer.fit(train_dataloader)` is deprecated in v1.4 and will be removed in v1.6.\"\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/configuration_validator.py:101: UserWarning: you defined a validation_step but have no val_dataloader. Skipping val loop\n",
      "  rank_zero_warn(f\"you defined a {step_name} but have no {loader_name}. Skipping {stage} loop\")\n",
      "2022-07-21 19:14:22,422 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "2022-07-21 19:14:22,422 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - distributed_backend=ddp_subprocess\n",
      "2022-07-21 19:14:22,423 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - All DDP processes registered. Starting ddp with 4 processes\n",
      "2022-07-21 19:14:22,424 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 2, MEMBER: 3/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 3, MEMBER: 4/4\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 4 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "Global seed set to 7\n",
      "Global seed set to 7\n",
      "Global seed set to 7\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.696    Total estimated model params size (MB)\n",
      "Global seed set to 7\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/98 [00:00<00:00, 3844.46it/s]  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 98/98 [01:03<00:00,  1.55it/s, loss=0.172, v_num=8]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n",
      "2022-07-21 19:46:32,559 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "2022-07-21 19:46:32,560 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - distributed_backend=ddp_subprocess\n",
      "2022-07-21 19:46:32,561 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - All DDP processes registered. Starting ddp with 4 processes\n",
      "2022-07-21 19:46:32,562 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32min 10s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "evaluate\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/4\n",
      "Global seed set to 7\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 3, MEMBER: 4/4\n",
      "initializing ddp: GLOBAL_RANK: 2, MEMBER: 3/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/4\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 4 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing: 100%|██████████| 98/98 [00:19<00:00,  6.48it/s]--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.8793799877166748, 'test_loss': 0.35129544138908386}\n",
      "--------------------------------------------------------------------------------\n",
      "Testing: 100%|██████████| 98/98 [00:19<00:00,  4.98it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet(learning_rate=0.1, num_processes=2)\n",
    "trainer = Trainer(max_epochs=30, \n",
    "                  num_processes=2,\n",
    "                  channels_last=True,\n",
    "                  fast_dev_run=DEV_RUN)\n",
    "fit_time_dit = %timeit -n 1 -r 1 -o \\\n",
    "trainer.fit(model, train_dataloaders=train_loader)\n",
    "metric_dit = trainer.test(model, dataloaders=test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Enable both distributed training and ipex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "2022-07-21 18:42:45,191 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "2022-07-21 18:42:45,192 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - distributed_backend=ddp_subprocess\n",
      "2022-07-21 18:42:45,194 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - All DDP processes registered. Starting ddp with 4 processes\n",
      "2022-07-21 18:42:45,194 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 2, MEMBER: 3/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 3, MEMBER: 4/4\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 4 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "Global seed set to 7\n",
      "Global seed set to 7\n",
      "Global seed set to 7\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.712    Total estimated model params size (MB)\n",
      "Global seed set to 7\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/98 [00:00<00:00, 2957.90it/s]  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "[W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:127: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 98/98 [01:02<00:00,  1.60it/s, loss=0.169, v_num=7]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n",
      "2022-07-21 19:13:59,243 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n",
      "2022-07-21 19:13:59,244 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - distributed_backend=ddp_subprocess\n",
      "2022-07-21 19:13:59,245 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - All DDP processes registered. Starting ddp with 4 processes\n",
      "2022-07-21 19:13:59,246 - bigdl.nano.pytorch.plugins.ddp_subprocess - INFO - ----------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31min 14s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n",
      "evaluate\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 3, MEMBER: 4/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 2, MEMBER: 3/4\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/4\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 4 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing: 100%|██████████| 98/98 [00:19<00:00,  6.66it/s]--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.9152399897575378, 'test_loss': 0.2421105057001114}\n",
      "--------------------------------------------------------------------------------\n",
      "Testing: 100%|██████████| 98/98 [00:19<00:00,  5.10it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPyTorch1.12/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet(learning_rate=0.1, num_processes=2)\n",
    "trainer = Trainer(max_epochs=30, \n",
    "                  num_processes=2,\n",
    "                  use_ipex=True,\n",
    "                  channels_last=True,\n",
    "                  fast_dev_run=DEV_RUN)\n",
    "fit_time_dit_ipex = %timeit -n 1 -r 1 -o \\\n",
    "trainer.fit(model, train_dataloaders=train_loader)\n",
    "metric_dit_ipex = trainer.test(model, dataloaders=test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "|      Precision    | Fit Time(s)         | Accuracy(%) |\n",
      "|        Basic      |       2977.52       |    0.90524    |\n",
      "|        IPEX       |       2789.57       |    0.89674    |\n",
      "|     Distributed   |       1930.14       |    0.87938    |\n",
      "|    DIST with IPEX |       1874.09       |    0.91524    |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "template = \"\"\"\n",
    "|      Precision    | Fit Time(s)         | Accuracy(%) |\n",
    "|        Basic      |       {:5.2f}       |    {:5.5f}    |\n",
    "|        IPEX       |       {:5.2f}       |    {:5.5f}    |\n",
    "|     Distributed   |       {:5.2f}       |    {:5.5f}    |\n",
    "|    DIST with IPEX |       {:5.2f}       |    {:5.5f}    |\n",
    "\"\"\"\n",
    "summary = template.format(\n",
    "    fit_time_basic.average, metric_basic[0]['test_acc'],\n",
    "    fit_time_ipex.average, metric_ipex[0]['test_acc'],\n",
    "    fit_time_dit.average, metric_dit[0]['test_acc'],\n",
    "    fit_time_dit_ipex.average, metric_dit_ipex[0]['test_acc']\n",
    ")\n",
    "print(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.10 ('nanoPyTorch1.12')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a267b6bb7281c99fd8dafc5773770b1464c39ade8a473a7fcbc55d05aea85c4c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
